{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initializing and Restoring EMI-RNN Graphs\n",
    "\n",
    "The *EMI-RNN* implementation supports four forms of initialization/restoring:\n",
    "1. An entirely new graph can be constructed with randomly initialized weights.\n",
    "2. A saved graph can be loaded into the current `EMI_Driver`.\n",
    "2. An entirely new graph can be constructed with weights initialized from a saved graph. This behavior is essentially restoration of a saved graph.\n",
    "3. (*Experimental*) Initializing/Restoring using numpy matrices. \n",
    "\n",
    "All three methods will be illustrated in this notebook. This notebook uses the HAR dataset and builds on the [EMI LSTM example.ipynb](00_emi_lstm_example.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:47:35.576928Z",
     "start_time": "2018-08-19T11:47:34.670184Z"
    }
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import os\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "# To include edgeml in python path\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] ='0'\n",
    "\n",
    "# MI-RNN and EMI-RNN imports\n",
    "from edgeml_tf.graph.rnn import EMI_DataPipeline\n",
    "from edgeml_tf.graph.rnn import EMI_BasicLSTM\n",
    "from edgeml_tf.trainer.emirnnTrainer import EMI_Trainer, EMI_Driver\n",
    "import edgeml_tf.utils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us set up some network parameters for the computation graph."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:47:35.590781Z",
     "start_time": "2018-08-19T11:47:35.578914Z"
    }
   },
   "outputs": [],
   "source": [
    "# Network parameters for our LSTM + FC Layer\n",
    "NUM_HIDDEN = 32\n",
    "NUM_TIMESTEPS = 48\n",
    "NUM_FEATS = 9\n",
    "FORGET_BIAS = 1.0\n",
    "NUM_OUTPUT = 6\n",
    "USE_DROPOUT = True\n",
    "KEEP_PROB = 0.75\n",
    "\n",
    "# For dataset API\n",
    "PREFETCH_NUM = 5\n",
    "BATCH_SIZE = 32\n",
    "\n",
    "# Number of epochs in *one iteration*\n",
    "NUM_EPOCHS = 3\n",
    "# Number of iterations in *one round*. After each iteration,\n",
    "# the model is dumped to disk. At the end of the current\n",
    "# round, the best model among all the dumped models in the\n",
    "# current round is picked up..\n",
    "NUM_ITER = 1\n",
    "# A round consists of multiple training iterations and a belief\n",
    "# update step using the best model from all of these iterations\n",
    "NUM_ROUNDS = 2\n",
    "\n",
    "# A staging direcory to store models\n",
    "MODEL_PREFIX = '/tmp/models/model-lstm'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:47:35.694831Z",
     "start_time": "2018-08-19T11:47:35.592516Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape is: (6220, 6, 48, 9)\n",
      "y_train shape is: (6220, 6, 6)\n",
      "x_test shape is: (1132, 6, 48, 9)\n",
      "y_test shape is: (1132, 6, 6)\n"
     ]
    }
   ],
   "source": [
    "# Loading the data\n",
    "x_train, y_train = np.load('./HAR/48_16/x_train.npy'), np.load('./HAR/48_16/y_train.npy')\n",
    "x_test, y_test = np.load('./HAR/48_16/x_test.npy'), np.load('./HAR/48_16/y_test.npy')\n",
    "x_val, y_val = np.load('./HAR/48_16/x_val.npy'), np.load('./HAR/48_16/y_val.npy')\n",
    "\n",
    "# BAG_TEST, BAG_TRAIN, BAG_VAL represent bag_level labels. These are used for the label update\n",
    "# step of EMI/MI RNN\n",
    "BAG_TEST = np.argmax(y_test[:, 0, :], axis=1)\n",
    "BAG_TRAIN = np.argmax(y_train[:, 0, :], axis=1)\n",
    "BAG_VAL = np.argmax(y_val[:, 0, :], axis=1)\n",
    "NUM_SUBINSTANCE = x_train.shape[1]\n",
    "print(\"x_train shape is:\", x_train.shape)\n",
    "print(\"y_train shape is:\", y_train.shape)\n",
    "print(\"x_test shape is:\", x_val.shape)\n",
    "print(\"y_test shape is:\", y_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:47:35.739003Z",
     "start_time": "2018-08-19T11:47:35.696723Z"
    }
   },
   "outputs": [],
   "source": [
    "# Define the linear secondary classifier\n",
    "def createExtendedGraph(self, baseOutput, *args, **kwargs):\n",
    "    W1 = tf.Variable(np.random.normal(size=[NUM_HIDDEN, NUM_OUTPUT]).astype('float32'), name='W1')\n",
    "    B1 = tf.Variable(np.random.normal(size=[NUM_OUTPUT]).astype('float32'), name='B1')\n",
    "    y_cap = tf.add(tf.tensordot(baseOutput, W1, axes=1), B1, name='y_cap_tata')\n",
    "    self.output = y_cap\n",
    "    self.graphCreated = True\n",
    "    \n",
    "def addExtendedAssignOps(self, graph, W_val=None, B_val=None):\n",
    "    W1 = graph.get_tensor_by_name('W1:0')\n",
    "    B1 = graph.get_tensor_by_name('B1:0')\n",
    "    W1_op = tf.assign(W1, W_val)\n",
    "    B1_op = tf.assign(B1, B_val)\n",
    "    self.assignOps.extend([W1_op, B1_op])\n",
    "\n",
    "def restoreExtendedGraph(self, graph, *args, **kwargs):\n",
    "    y_cap = graph.get_tensor_by_name('y_cap_tata:0')\n",
    "    self.output = y_cap\n",
    "    self.graphCreated = True\n",
    "    \n",
    "def feedDictFunc(self, keep_prob, **kwargs):\n",
    "    feedDict = {self._emiGraph.keep_prob: keep_prob}\n",
    "    return feedDict\n",
    "    \n",
    "EMI_BasicLSTM._createExtendedGraph = createExtendedGraph\n",
    "EMI_BasicLSTM._restoreExtendedGraph = restoreExtendedGraph\n",
    "EMI_BasicLSTM.addExtendedAssignOps = addExtendedAssignOps\n",
    "\n",
    "if USE_DROPOUT is True:\n",
    "    EMI_Driver.feedDictFunc = feedDictFunc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T09:34:06.288012Z",
     "start_time": "2018-08-19T09:34:06.285286Z"
    }
   },
   "source": [
    "## 1. Initializing a New Computation Graph\n",
    "\n",
    "In the most common use cases, a new EMI-RNN graph would be created and trained"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:49:55.326002Z",
     "start_time": "2018-08-19T11:49:50.568621Z"
    }
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "inputPipeline = EMI_DataPipeline(NUM_SUBINSTANCE, NUM_TIMESTEPS, NUM_FEATS, NUM_OUTPUT)\n",
    "emiLSTM = EMI_BasicLSTM(NUM_SUBINSTANCE, NUM_HIDDEN, NUM_TIMESTEPS, NUM_FEATS,\n",
    "                        forgetBias=FORGET_BIAS, useDropout=USE_DROPOUT)\n",
    "emiTrainer = EMI_Trainer(NUM_TIMESTEPS, NUM_OUTPUT, lossType='xentropy')\n",
    "\n",
    "# Construct the graph\n",
    "g1 = tf.Graph()    \n",
    "with g1.as_default():\n",
    "    x_batch, y_batch = inputPipeline()\n",
    "    y_cap = emiLSTM(x_batch)\n",
    "    emiTrainer(y_cap, y_batch)\n",
    "    \n",
    "with g1.as_default():\n",
    "    emiDriver = EMI_Driver(inputPipeline, emiLSTM, emiTrainer)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lets initialize a new session with this graph and train a model. The saved model will be used later for restoring."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:48:32.894784Z",
     "start_time": "2018-08-19T11:47:41.258027Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Update policy: top-k\n",
      "Training with MI-RNN loss for 1 rounds\n",
      "Round: 0\n",
      "Epoch   2 Batch   180 (  570) Loss 0.00457 Acc 0.88542 | Val acc 0.94594 | Model saved to /tmp/models/model-lstm, global_step 1000\n",
      "INFO:tensorflow:Restoring parameters from /tmp/models/model-lstm-1000\n",
      "Round: 1\n",
      "Switching to EMI-Loss function\n",
      "Epoch   2 Batch   180 (  570) Loss 0.30313 Acc 0.93229 | Val acc 0.97439 | Model saved to /tmp/models/model-lstm, global_step 1001\n",
      "INFO:tensorflow:Restoring parameters from /tmp/models/model-lstm-1001\n"
     ]
    }
   ],
   "source": [
    "emiDriver.initializeSession(g1)\n",
    "y_updated, modelStats = emiDriver.run(numClasses=NUM_OUTPUT, x_train=x_train,\n",
    "                                      y_train=y_train, bag_train=BAG_TRAIN,\n",
    "                                      x_val=x_val, y_val=y_val, bag_val=BAG_VAL,\n",
    "                                      numIter=NUM_ITER, keep_prob=KEEP_PROB,\n",
    "                                      numRounds=NUM_ROUNDS, batchSize=BATCH_SIZE,\n",
    "                                      numEpochs=NUM_EPOCHS, modelPrefix=MODEL_PREFIX,\n",
    "                                      fracEMI=0.5, updatePolicy='top-k', k=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As the output above indicates, the last restored model is `/tmp/model-lstm-1001`. That is, with `MODEL_PREFIX = '/tmp/model-lstm'` and `GLOBAL_STEP=1001`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:48:33.294431Z",
     "start_time": "2018-08-19T11:48:32.897376Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy at k = 2: 0.919919\n"
     ]
    }
   ],
   "source": [
    "def earlyPolicy_minProb(instanceOut, minProb, **kwargs):\n",
    "    assert instanceOut.ndim == 2\n",
    "    classes = np.argmax(instanceOut, axis=1)\n",
    "    prob = np.max(instanceOut, axis=1)\n",
    "    index = np.where(prob >= minProb)[0]\n",
    "    if len(index) == 0:\n",
    "        assert (len(instanceOut) - 1) == (len(classes) - 1)\n",
    "        return classes[-1], len(instanceOut) - 1\n",
    "    index = index[0]\n",
    "    return classes[index], index\n",
    "\n",
    "\n",
    "k = 2\n",
    "predictions, predictionStep = emiDriver.getInstancePredictions(x_test, y_test, earlyPolicy_minProb,\n",
    "                                                               minProb=0.99, keep_prob=1.0)\n",
    "bagPredictions = emiDriver.getBagPredictions(predictions, minSubsequenceLen=k, numClass=NUM_OUTPUT)\n",
    "print('Accuracy at k = %d: %f' % (k,  np.mean((bagPredictions == BAG_TEST).astype(int))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Loading a Saved Graph into EMI-Driver\n",
    "\n",
    "We will reset the computation graph, load a saved graph into the current `EMI_Driver` and verify its outputs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:48:38.811810Z",
     "start_time": "2018-08-19T11:48:33.296990Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /tmp/models/model-lstm-1001\n",
      "Accuracy at k = 2: 0.919919\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "emiDriver.loadSavedGraphToNewSession(MODEL_PREFIX, 1001)\n",
    "k = 2\n",
    "predictions, predictionStep = emiDriver.getInstancePredictions(x_test, y_test, earlyPolicy_minProb,\n",
    "                                                               minProb=0.99, keep_prob=1.0)\n",
    "bagPredictions = emiDriver.getBagPredictions(predictions, minSubsequenceLen=k, numClass=NUM_OUTPUT)\n",
    "print('Accuracy at k = %d: %f' % (k,  np.mean((bagPredictions == BAG_TEST).astype(int))))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Initializing using a Saved Graph\n",
    "\n",
    "Here we will construct a new computation graph, but will use a previously trained model to initialize it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:48:38.818037Z",
     "start_time": "2018-08-19T11:48:38.814131Z"
    }
   },
   "outputs": [],
   "source": [
    "# Making sure the old graph and sessions are closed\n",
    "sess = emiDriver.getCurrentSession()\n",
    "sess.close()\n",
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the `GraphManager` to load the saved graph and load it into a new session."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:48:43.329691Z",
     "start_time": "2018-08-19T11:48:38.819945Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /tmp/models/model-lstm-1001\n"
     ]
    }
   ],
   "source": [
    "sess = tf.Session()\n",
    "graphManager = edgeml.utils.GraphManager()\n",
    "graph = graphManager.loadCheckpoint(sess, MODEL_PREFIX, globalStep=1001)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Construct the forward graph as before, but provide the loaded `graph` as an argument to `__init__`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:48:43.410591Z",
     "start_time": "2018-08-19T11:48:43.337841Z"
    }
   },
   "outputs": [],
   "source": [
    "inputPipeline = EMI_DataPipeline(NUM_SUBINSTANCE, NUM_TIMESTEPS, NUM_FEATS, NUM_OUTPUT, graph=graph)\n",
    "emiLSTM = EMI_BasicLSTM(NUM_SUBINSTANCE, NUM_HIDDEN, NUM_TIMESTEPS, NUM_FEATS,\n",
    "                        forgetBias=FORGET_BIAS, useDropout=USE_DROPOUT, graph=graph)\n",
    "emiTrainer = EMI_Trainer(NUM_TIMESTEPS, NUM_OUTPUT, lossType='xentropy', graph=graph)\n",
    "\n",
    "g1 = graph\n",
    "with g1.as_default():\n",
    "    x_batch, y_batch = inputPipeline()\n",
    "    y_cap = emiLSTM(x_batch)\n",
    "    emiTrainer(y_cap, y_batch)\n",
    "    \n",
    "with g1.as_default():\n",
    "    emiDriver = EMI_Driver(inputPipeline, emiLSTM, emiTrainer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let `EMI_Driver` know that we already have a session in place."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:48:43.424410Z",
     "start_time": "2018-08-19T11:48:43.421920Z"
    }
   },
   "outputs": [],
   "source": [
    "emiDriver.setSession(sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:48:44.324735Z",
     "start_time": "2018-08-19T11:48:43.426174Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy at k = 2: 0.919919\n"
     ]
    }
   ],
   "source": [
    "k = 2\n",
    "predictions, predictionStep = emiDriver.getInstancePredictions(x_test, y_test, earlyPolicy_minProb,\n",
    "                                                               minProb=0.99, keep_prob=1.0)\n",
    "bagPredictions = emiDriver.getBagPredictions(predictions, minSubsequenceLen=k, numClass=NUM_OUTPUT)\n",
    "print('Accuracy at k = %d: %f' % (k,  np.mean((bagPredictions == BAG_TEST).astype(int))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Restoring from Numpy Matrices\n",
    "\n",
    "We first extract the model matrices from the graph and dump it into `.npy` files. Then we load it back again and initialize a new graph with these matrices."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:48:44.379901Z",
     "start_time": "2018-08-19T11:48:44.326706Z"
    }
   },
   "outputs": [],
   "source": [
    "graph = tf.get_default_graph()\n",
    "W1 = graph.get_tensor_by_name('W1:0')\n",
    "B1 = graph.get_tensor_by_name('B1:0')\n",
    "allVars = emiLSTM.varList + [W1, B1]\n",
    "sess = emiDriver.getCurrentSession()\n",
    "allVars = sess.run(allVars)\n",
    "\n",
    "base = '/tmp/models/'\n",
    "np.save(base + 'kernel.npy', allVars[0])\n",
    "np.save(base + 'bias.npy', allVars[1])\n",
    "np.save(base + 'W1.npy', allVars[2])\n",
    "np.save(base + 'B1.npy', allVars[3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:16:55.606967Z",
     "start_time": "2018-08-19T11:16:55.535964Z"
    }
   },
   "source": [
    "Reset the current session and graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:48:44.389724Z",
     "start_time": "2018-08-19T11:48:44.381802Z"
    }
   },
   "outputs": [],
   "source": [
    "sess = emiDriver.getCurrentSession()\n",
    "sess.close()\n",
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the numpy matrices that will be used to initialize the graph."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:48:44.442241Z",
     "start_time": "2018-08-19T11:48:44.391384Z"
    }
   },
   "outputs": [],
   "source": [
    "base = '/tmp/models/'\n",
    "kernel = np.load(base + 'kernel.npy')\n",
    "bias = np.load(base + 'bias.npy')\n",
    "W = np.load(base + 'W1.npy')\n",
    "B = np.load(base + 'B1.npy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:38:36.118298Z",
     "start_time": "2018-08-19T11:38:36.113810Z"
    }
   },
   "source": [
    "Proceed with graph construction as normally done, except that we add the requisite assignment operations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-19T11:48:51.378377Z",
     "start_time": "2018-08-19T11:48:44.444182Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PART IV: Accuracy at k = 2: 0.919919\n"
     ]
    }
   ],
   "source": [
    "inputPipeline = EMI_DataPipeline(NUM_SUBINSTANCE, NUM_TIMESTEPS, NUM_FEATS,\n",
    "                                 NUM_OUTPUT)\n",
    "emiLSTM = EMI_BasicLSTM(NUM_SUBINSTANCE, NUM_HIDDEN, NUM_TIMESTEPS, NUM_FEATS,\n",
    "                        forgetBias=FORGET_BIAS, useDropout=USE_DROPOUT)\n",
    "emiTrainer = EMI_Trainer(NUM_TIMESTEPS, NUM_OUTPUT, lossType='xentropy')\n",
    "\n",
    "tf.reset_default_graph()\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "    x_batch, y_batch = inputPipeline()\n",
    "    y_cap = emiLSTM(x_batch)\n",
    "    emiTrainer(y_cap, y_batch)\n",
    "    # Add the assignment operations\n",
    "    emiLSTM.addBaseAssignOps(graph, [kernel, bias])\n",
    "    emiLSTM.addExtendedAssignOps(graph, W, B)\n",
    "    # Setup the driver. You can run the initializations manually as well\n",
    "    emiDriver = EMI_Driver(inputPipeline, emiLSTM, emiTrainer)\n",
    "\n",
    "emiDriver.initializeSession(graph)\n",
    "# Run the assignment operations\n",
    "sess = emiDriver.getCurrentSession()\n",
    "sess.run(emiLSTM.assignOps)\n",
    "\n",
    "k = 2\n",
    "predictions, predictionStep = emiDriver.getInstancePredictions(x_test, y_test,\n",
    "                                                               earlyPolicy_minProb,\n",
    "                                                               minProb=0.99,\n",
    "                                                               keep_prob=1.0)\n",
    "bagPredictions = emiDriver.getBagPredictions(predictions, minSubsequenceLen=k,\n",
    "                                             numClass=NUM_OUTPUT)\n",
    "print('PART IV: Accuracy at k = %d: %f' % (k,  np.mean((bagPredictions ==\n",
    "                                                        BAG_TEST).astype(int))))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
